import os
from pet_spider import PetSpider

if __name__ == '__main__':
    url = 'http://www.news.cn/politics/xxjxs/index.htm'
    url_prefix = 'http://www.news.cn/politics/xxjxs/index.htm'
    image_path = 'cats'
    html_path = 'html'
    url_path = 'url'

    if not os.path.isdir(image_path):
        os.makedirs(image_path)
    if not os.path.isdir(url_path):
        os.makedirs(html_path)
    if not os.path.isdir(url_path):
        os.makedirs(url_path)

    pet_spider = PetSpider(image_path, html_path, url_path)
    print('Download webpages.')
    pet_spider.get_html_content(url, url_prefix)
    print('Extract urls.')
    pet_spider.get_urls()
    print('Download images')
pet_spider.download_images()
